{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['.ipynb_checkpoints',\n",
       " 'baseline_.py',\n",
       " 'base_helper.py',\n",
       " 'data_model.ipynb',\n",
       " 'feature_analysis.tex',\n",
       " 'my_transction_analysis.ipynb',\n",
       " 'opration__analysis.ipynb',\n",
       " 'op_data_analysis.ipynb',\n",
       " 'op_data_analysis.py',\n",
       " 'outliers_detection.py',\n",
       " 'test_train_whole_tr1.ipynb',\n",
       " 'train_op_feature.csv',\n",
       " 'train_tst_feature.csv',\n",
       " 'transcation_data_analysis.ipynb',\n",
       " 'transction_analysis.ipynb',\n",
       " 'Untitled.ipynb',\n",
       " '__init__.py',\n",
       " '__pycache__',\n",
       " '相同字段分析.ipynb']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "from base_helper import *\n",
    "import pandas as pd\n",
    "tag = get_tag_train_new()\n",
    "import os\n",
    "os.listdir('./')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = pd.read_csv('./train_op_feature.csv')\n",
    "df2 = pd.read_csv('./train_tst_feature.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(29728, 102) (30542, 113)\n"
     ]
    }
   ],
   "source": [
    "print(df1.shape, df2.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "29091"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(df1.UID.unique().tolist()) & set(df2.UID.unique().tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_feature = pd.merge(df1, df2, on='UID', how='inner')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_feature = pd.merge(df_feature, tag, on='UID', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(29091, 215)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_feature.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>Tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>mac_loss_tst</td>\n",
       "      <td>-0.361106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>avg_day_mode_1</td>\n",
       "      <td>-0.281245</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>wifi</td>\n",
       "      <td>-0.181658</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>mode_1</td>\n",
       "      <td>-0.129309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>avg_day_mode_13</td>\n",
       "      <td>-0.109486</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>avg_day_mode_12</td>\n",
       "      <td>-0.108801</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>avg_op_mode_13</td>\n",
       "      <td>-0.106699</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>avg_day_mode_14</td>\n",
       "      <td>-0.106464</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>avg_op_mode_12</td>\n",
       "      <td>-0.103865</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>avg_op_mode_14</td>\n",
       "      <td>-0.102505</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>success_1</td>\n",
       "      <td>-0.099198</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>mode_-1</td>\n",
       "      <td>-0.099064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>mode_13</td>\n",
       "      <td>-0.098735</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>mode_12</td>\n",
       "      <td>-0.097858</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>avg_day_geo</td>\n",
       "      <td>-0.096685</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>mode_14</td>\n",
       "      <td>-0.096213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>trans_mer_number_min</td>\n",
       "      <td>-0.092445</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>avg_day_wifi_day</td>\n",
       "      <td>-0.092160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>ip_number</td>\n",
       "      <td>-0.087567</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>avg_op_time_op_2</td>\n",
       "      <td>-0.087514</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>wifi_day</td>\n",
       "      <td>-0.082198</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>trans_type2_max_105</td>\n",
       "      <td>-0.081946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>avg_day_time_op_2</td>\n",
       "      <td>-0.081004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>trans_mer_number_mean</td>\n",
       "      <td>-0.076125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>avg_day_mode_6</td>\n",
       "      <td>-0.068028</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>time_op_2</td>\n",
       "      <td>-0.067725</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>avg_day_mode_7</td>\n",
       "      <td>-0.066816</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>trans_type2_mean_105</td>\n",
       "      <td>-0.066190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>avg_day_mode_8</td>\n",
       "      <td>-0.066069</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>device_day</td>\n",
       "      <td>-0.063638</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>185</th>\n",
       "      <td>mode_4</td>\n",
       "      <td>0.149667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>186</th>\n",
       "      <td>market_type_max_2.0</td>\n",
       "      <td>0.150605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>187</th>\n",
       "      <td>avg_op_mode_4</td>\n",
       "      <td>0.151536</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>188</th>\n",
       "      <td>avg_op_mode_2</td>\n",
       "      <td>0.152507</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>189</th>\n",
       "      <td>avg_op_mode_11</td>\n",
       "      <td>0.153106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>190</th>\n",
       "      <td>avg_day_mode_11</td>\n",
       "      <td>0.161048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191</th>\n",
       "      <td>avg_day_os_day</td>\n",
       "      <td>0.174891</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>192</th>\n",
       "      <td>market_type_mean_2.0</td>\n",
       "      <td>0.180957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>193</th>\n",
       "      <td>wefi_loss</td>\n",
       "      <td>0.181603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>194</th>\n",
       "      <td>avg_day_trans_type2</td>\n",
       "      <td>0.182852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>195</th>\n",
       "      <td>market_type_min_2.0</td>\n",
       "      <td>0.195164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>196</th>\n",
       "      <td>mode_3</td>\n",
       "      <td>0.208344</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>197</th>\n",
       "      <td>avg_day_mode_3</td>\n",
       "      <td>0.210697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>198</th>\n",
       "      <td>avg_op_os_day</td>\n",
       "      <td>0.218290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199</th>\n",
       "      <td>amt_channel_mean_118</td>\n",
       "      <td>0.223121</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>200</th>\n",
       "      <td>avg_day_trans_type1</td>\n",
       "      <td>0.236699</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201</th>\n",
       "      <td>avg_day_ip_tst</td>\n",
       "      <td>0.242344</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202</th>\n",
       "      <td>geo_loss</td>\n",
       "      <td>0.271026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>203</th>\n",
       "      <td>avg_day_merchant</td>\n",
       "      <td>0.275808</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>204</th>\n",
       "      <td>channel_count</td>\n",
       "      <td>0.277052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205</th>\n",
       "      <td>avg_op_mode_3</td>\n",
       "      <td>0.280170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206</th>\n",
       "      <td>is_apple</td>\n",
       "      <td>0.289128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207</th>\n",
       "      <td>amt_channel_min_118</td>\n",
       "      <td>0.292527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208</th>\n",
       "      <td>avg_day_geo_tst</td>\n",
       "      <td>0.329388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>209</th>\n",
       "      <td>is_apple_tst</td>\n",
       "      <td>0.338244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>210</th>\n",
       "      <td>avg_day_channel</td>\n",
       "      <td>0.342996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>211</th>\n",
       "      <td>ip_loss_tst</td>\n",
       "      <td>0.462252</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>212</th>\n",
       "      <td>Tag</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213</th>\n",
       "      <td>day_shift_min</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>214</th>\n",
       "      <td>avg_trans_ip1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>215 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                     index       Tag\n",
       "0             mac_loss_tst -0.361106\n",
       "1           avg_day_mode_1 -0.281245\n",
       "2                     wifi -0.181658\n",
       "3                   mode_1 -0.129309\n",
       "4          avg_day_mode_13 -0.109486\n",
       "5          avg_day_mode_12 -0.108801\n",
       "6           avg_op_mode_13 -0.106699\n",
       "7          avg_day_mode_14 -0.106464\n",
       "8           avg_op_mode_12 -0.103865\n",
       "9           avg_op_mode_14 -0.102505\n",
       "10               success_1 -0.099198\n",
       "11                 mode_-1 -0.099064\n",
       "12                 mode_13 -0.098735\n",
       "13                 mode_12 -0.097858\n",
       "14             avg_day_geo -0.096685\n",
       "15                 mode_14 -0.096213\n",
       "16    trans_mer_number_min -0.092445\n",
       "17        avg_day_wifi_day -0.092160\n",
       "18               ip_number -0.087567\n",
       "19        avg_op_time_op_2 -0.087514\n",
       "20                wifi_day -0.082198\n",
       "21     trans_type2_max_105 -0.081946\n",
       "22       avg_day_time_op_2 -0.081004\n",
       "23   trans_mer_number_mean -0.076125\n",
       "24          avg_day_mode_6 -0.068028\n",
       "25               time_op_2 -0.067725\n",
       "26          avg_day_mode_7 -0.066816\n",
       "27    trans_type2_mean_105 -0.066190\n",
       "28          avg_day_mode_8 -0.066069\n",
       "29              device_day -0.063638\n",
       "..                     ...       ...\n",
       "185                 mode_4  0.149667\n",
       "186    market_type_max_2.0  0.150605\n",
       "187          avg_op_mode_4  0.151536\n",
       "188          avg_op_mode_2  0.152507\n",
       "189         avg_op_mode_11  0.153106\n",
       "190        avg_day_mode_11  0.161048\n",
       "191         avg_day_os_day  0.174891\n",
       "192   market_type_mean_2.0  0.180957\n",
       "193              wefi_loss  0.181603\n",
       "194    avg_day_trans_type2  0.182852\n",
       "195    market_type_min_2.0  0.195164\n",
       "196                 mode_3  0.208344\n",
       "197         avg_day_mode_3  0.210697\n",
       "198          avg_op_os_day  0.218290\n",
       "199   amt_channel_mean_118  0.223121\n",
       "200    avg_day_trans_type1  0.236699\n",
       "201         avg_day_ip_tst  0.242344\n",
       "202               geo_loss  0.271026\n",
       "203       avg_day_merchant  0.275808\n",
       "204          channel_count  0.277052\n",
       "205          avg_op_mode_3  0.280170\n",
       "206               is_apple  0.289128\n",
       "207    amt_channel_min_118  0.292527\n",
       "208        avg_day_geo_tst  0.329388\n",
       "209           is_apple_tst  0.338244\n",
       "210        avg_day_channel  0.342996\n",
       "211            ip_loss_tst  0.462252\n",
       "212                    Tag  1.000000\n",
       "213          day_shift_min       NaN\n",
       "214          avg_trans_ip1       NaN\n",
       "\n",
       "[215 rows x 2 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_feature.corr()['Tag'].sort_values().reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
